package main

import (
	"fmt"
	"github.com/steelx/extractlinks"
	"log"
	"net/http"
	"os"
)

var tokens = make(chan struct{}, 20)

func crawl(url string) []string {
	fmt.Println(url)
	tokens <- struct{}{} // acquire a token
	resp, _ := http.Get(url)
	fmt.Println(resp)
	links, err := extractlinks.All(resp.Body)
	if err != nil {
		log.Print(err)
	}
	<-tokens // release the token
	var list []string
	for _, link := range links {
		list = append(list, link.Href)
	}
	return list
}

func main() {
	worklist := make(chan []string)
	var n int // number of pending sends to worklist
	// Start with the command-line arguments.
	n++
	go func() { worklist <- os.Args[1:] }()
	// Crawl the web concurrently.
	seen := make(map[string]bool)
	for ; n > 0; n-- {
		list := <-worklist
		for _, link := range list {
			if !seen[link] {
				seen[link] = true
				n++
				go func(link string) {
					worklist <- crawl(link)
				}(link)
			}
		}
	}

}
